package com.qf.index;

import cn.hutool.core.collection.CollUtil;
import cn.hutool.core.collection.ConcurrentHashSet;
import cn.hutool.core.util.NumberUtil;
import cn.hutool.core.util.ZipUtil;
import cn.hutool.crypto.SecureUtil;
import cn.hutool.db.Db;
import cn.hutool.db.Entity;
import cn.hutool.json.JSON;
import cn.hutool.json.JSONArray;
import cn.hutool.json.JSONUtil;
import com.alibaba.fastjson.JSONObject;
import org.jsoup.Jsoup;

import java.io.IOException;
import java.sql.SQLException;
import java.text.SimpleDateFormat;
import java.util.*;
import java.util.concurrent.ConcurrentHashMap;
import java.util.regex.Pattern;

public class WebSite {
    public static void main(String[] args) throws SQLException, IOException {

        String newDataName = "scholar_25_02.";
        String doiTable = newDataName + "doi_location_20230112";
        String doiAddTable = newDataName + "doi_location_20230112_add";
        String summaryTable = newDataName + "unified_search_summary_0307";
        String pmArticleTable = newDataName + "spider_pubmed_vip_extend";
        String ncTextTable = "html.nc_text_table";
        String standardUnitSumTable = newDataName + "basic_standard_unit_sum";
        String pmDataAnalysisTable = newDataName + "pm_data_analysis_fund_unit";
        String basicCoreTable = newDataName + "basic_core_paper";
        String citationTable = newDataName + "basic_core_paper_citation";
        String referenceTable = newDataName + "basic_core_paper_reference_id";
        String titleTransTable = newDataName + "basic_fund_guidelines_title_trans";
        String summaryFinalTable = newDataName + "unified_search_summary_0309";
        String bibPmTable = newDataName + "bibliometric_pm_0321";
        String bibAidTable = newDataName + "bibliometric_aid_0321";
        String ncTextTopTable = newDataName + "nc_text_table_top_0321";
        String abstractTransTable = newDataName + "basic_abstract_trans";
        Map<Integer, byte[]> aidAndKeyWordMap = new HashMap<>();
        Map<String, Integer> termAndSumMap = new ConcurrentHashMap<>();
        Map<Integer, Map<Integer, Map<Integer, String>>> aidTypeIdUrlMap = new ConcurrentHashMap<>();

        //        String limit = "";
        String limit = " limit 100";

        updateWebSite(summaryTable, doiTable, doiAddTable, pmArticleTable, ncTextTable, limit, standardUnitSumTable, pmDataAnalysisTable, basicCoreTable, citationTable, referenceTable, titleTransTable, aidAndKeyWordMap, summaryFinalTable, bibPmTable, bibAidTable, ncTextTopTable, termAndSumMap,abstractTransTable);


// main
    }


    public static void updateWebSite(String summaryTable, String doiTable, String doiAddTable, String pmArticleTable, String ncTextTable, String limit, String standardUnitSumTable, String pmDataAnalysisTable, String basicCoreTable,
                                     String citationTable, String referenceTable, String titleTransTable, Map<Integer, byte[]> aidAndKeyWordMap, String summaryFinalTable, String bibPmTable, String bibAidTable, String ncTextTopTable,
                                     Map<String, Integer> termAndSumMap,String abstractTransTable) throws SQLException {
        SimpleDateFormat format = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");

        Db.use().execute("truncate table " + referenceTable);
        insertReferenceTable(citationTable, "id", "pid`,`citation_id", referenceTable, limit);
        System.out.println("\033[31;1m" + format.format(new Date()) + " ⒁ 更新被引，表名：" + citationTable + Utils.printMemoryStr() + "\033[0m");

        Map<String, String> doiAndLocationMap = new ConcurrentHashMap<>();
        Map<Integer, byte[]> aidAndTiAuJ9Map = new ConcurrentHashMap<>();
        WebSite.getDoiAndLocationMap(doiTable, "id", "doi`,`location", doiAndLocationMap, limit);
        WebSite.getDoiAndLocationMap(doiAddTable, "id", "doi`,`location", doiAndLocationMap, limit);
        System.out.println("\033[31;1m" + format.format(new Date()) + " ⒃ 读取doi的下一跳结束，doiAndLocationMap.size()：" + doiAndLocationMap.size() + Utils.printMemoryStr() + "\033[0m");

        Map<Integer, byte[]> pmIdAndAdListMap = getPmIdAndAdListMap(pmArticleTable, "id", "pmid`,`author_extend", limit);
        System.out.println("\033[31;1m" + format.format(new Date()) + " ⒄ 读取pm的ad信息结束，pmIdAndAdListMap.size()：" + pmIdAndAdListMap.size() + Utils.printMemoryStr() + "\033[0m");

        Map<Integer, String> aidAndReferencesMap = getAidAndReferencesMap(summaryTable, "id", "aid`,`year`,`auth`,`journal`,`title`,`cn_ti`,`info`,`pmId", limit, aidAndTiAuJ9Map);
        System.out.println("\033[31;1m" + format.format(new Date()) + " ⒅ 读取参文信息结束，aidAndReferencesMap.size()：" + aidAndReferencesMap.size() + " aidAndTiAuJ9Map.size()：" + aidAndTiAuJ9Map.size() + Utils.printMemoryStr() + "\033[0m");

        Set<Integer> bookChapterSet = getBookChapterSet(ncTextTable, "id", "pmid`,`ptList", limit);
        System.out.println("\033[31;1m" + format.format(new Date()) + " ⒅ 读取Book Chapter结束，bookChapterSet.size()：" + bookChapterSet.size() + Utils.printMemoryStr() + "\033[0m");

        // 老鲍合并表（basic_core_paper）中，有ut与pm相同的（比如：001286151200001->39559033），但在ut表（spider_tb_true_92007022_1125）中，却没有pm（即ut认为此ut与pm不是同一篇，或者没有及时更新），此类的约有35万余条，给予补上。
        Map<String, Integer> utMapPmIdMap = getUtMapPmIdMap(basicCoreTable, "id", "pm`,`wos_ut", limit);
        System.out.println("\033[31;1m" + format.format(new Date()) + " ⒆ 读取ut与pmId的映射关系结束，utMapPmIdMap.size()：" + utMapPmIdMap.size() + Utils.printMemoryStr() + "\033[0m");

        Map<Integer, byte[]> aidAndCitationMap = getAidAndCitationMap(citationTable, "id", "pid`,`citation_id", "citation", limit);
        Map<Integer, byte[]> aidAndReferenceMap = getAidAndCitationMap(referenceTable, "id", "pid`,`reference_id", "reference", limit);
        System.out.println("\033[31;1m" + format.format(new Date()) + " ⒆ 读取参文与引文结束，aidAndCitationMap.size()：" + aidAndCitationMap.size() + " aidAndReferenceMap.size()：" + aidAndReferenceMap.size() + Utils.printMemoryStr() + "\033[0m");

        // 这里更新了七种：预警期刊、参考文献、被引次数、发文国家集合、是否中国人发文、全文网址域名、pm和ad集合。
        String field = "cscd_id`,`ncbi_id`,`yiigle_id`,`wos_id`,`weipu_id`,`wanfang_id`,`cnki_id`,`key_word_list`,`journal`,`aid`,`citation_quota`,`reference_quota`,`ti_key`,`ab_key`,`kw_key`,`abb_word`,`zky_dalei`,`jcr_if`,`year`,`title`,`keyword`,`ab`,`mesh`,`jid`,`discussion`," +
                "`results`,`methods`,`background`,`c1_au`,`rp_au`,`cite`,`cn_ti`,`cn_ab`,`cn_kw`,`cn_type`,`pt`,`is_cns`,`pmId`,`ut`,`doi`,`nlmId`,`cite_score`,`jcr_quartile`,`fund_list`,`drug_gene_pathway_disease`,`auth`,`info`,`is_retraction`,`is_china`,`cn_discussion`,`cn_results`," +
                "`cn_methods`,`cn_background`,`cs_sjr`,`is_beida`,`is_cs`,`is_cscd`,`is_jcr`,`is_pubmed`,`is_tjy`,`is_yiigle`,`is_zky`,`zky_subject`,`is_core`,`mail_list`,`subject`,`kw_plus`,`times_cited`,`language`,`references`,`citations`,`is_china_hosp`,`is_guidelines`," +
                "`is_pubmed_article`,`is_6year`,`pm_ad`,`ncbi_subject`,`ncbi_journal`,`ncbi_mesh`,`web_site`,`cn_country`,`early_warning`,`is_fund`,`fund_auth`,`url";
        updateReferences(summaryTable, "id", field, aidAndReferencesMap, limit, pmIdAndAdListMap,
                doiAndLocationMap, aidAndTiAuJ9Map, bookChapterSet, standardUnitSumTable, pmDataAnalysisTable, utMapPmIdMap, aidAndCitationMap, aidAndReferenceMap, aidAndKeyWordMap, summaryFinalTable, termAndSumMap);
        System.out.println("\033[31;1m" + format.format(new Date()) + " ⒇ 更新参文等七项数据结束" + Utils.printMemoryStr() + "\033[0m");
        utMapPmIdMap.clear();
        doiAndLocationMap.clear();
        aidAndTiAuJ9Map.clear();
        pmIdAndAdListMap.clear();
        aidAndReferencesMap.clear();
        aidAndCitationMap.clear();
        aidAndReferenceMap.clear();
        termAndSumMap.clear();
        bookChapterSet.clear();
        aidAndKeyWordMap.clear();

        TopArticle.readTable(ncTextTable, "id", "ta`,`jid`,`dp", ncTextTopTable);
        System.out.println("\033[31;1m" + format.format(new Date()) + " ⒇ 产生TOP期刊论文结束，此程序需要两个TOP期刊清单。" + Utils.printMemoryStr() + "\033[0m");

        field = "drug_gene_pathway_disease`,`c1_au`,`rp_au`,`citations`,`pmId`,`doi`,`mesh`,`ab`,`keyword`,`title`,`ncbi_journal`,`year`,`is_cns`,`pt`,`is_china`,`nlmId`,`cn_country`,`info`,`ncbi_mesh`,`ncbi_subject`," +
                "`jcr_if`,`cite_score`,`zky_dalei`,`jcr_quartile`,`web_site`,`language`,`subject`,`ti_key`,`ab_key`,`kw_key`,`fund_list`,`times_cited`,`references`,`kw_plus`,`aid`,`ut`,`journal";
        Bibliometric.insertBibTable(summaryFinalTable, "id", field, bibPmTable, limit, bibAidTable);
        System.out.println("\033[31;1m" + format.format(new Date()) + " ⒇ 小葛使用的bib的两个表结束。" + Utils.printMemoryStr() + "\033[0m");

        Map<String, byte[]> titleTransMap = getTitleTransMap(titleTransTable, "id", "original`,`trans", limit);
        System.out.println("\033[31;1m" + format.format(new Date()) + " ⒇ 获取已经翻译的中文标题结束，下面开始进行有道翻译。titleTransMap.size()：" + titleTransMap.size() + Utils.printMemoryStr() + "\033[0m");
        transTitle(summaryFinalTable, "id", "year`,`pmId`,`ut`,`title`,`cn_ti", 0, 100, titleTransMap, titleTransTable);
        titleTransMap.clear();

        Map<String, byte[]> abstractTransMap = WebSite.getTitleTransMap(abstractTransTable, "id", "original`,`trans", limit);
        System.out.println("\033[31;1m" + format.format(new Date()) + " ⒇ 获取已经翻译的摘要结束，下面开始进行有道翻译。abstractTransMap.size()：" + abstractTransMap.size() + Utils.printMemoryStr() + "\033[0m");
        transAb.updateAbstract(summaryFinalTable, "id", "aid`,`discussion`,`results`,`methods`,`background`,`cn_discussion`,`cn_results`,`cn_methods`,`cn_background`,`pmId`,`ut`,`year", abstractTransMap, abstractTransTable);
        abstractTransMap.clear();


    }

    public static void updateReferences(String summaryTable, String autoId, String field, Map<Integer, String> aidAndReferencesMap, String limit, Map<Integer, byte[]> pmIdAndAdListMap, Map<String, String> doiAndLocationMap, Map<Integer, byte[]> aidAndTiAuJ9Map,
                                        Set<Integer> bookChapterSet, String standardUnitSumTable, String pmDataAnalysisTable, Map<String, Integer> utMapPmIdMap, Map<Integer, byte[]> aidAndCitationMap, Map<Integer, byte[]> aidAndReferenceMap, Map<Integer, byte[]> aidAndKeyWordMap,
                                        String summaryFinalTable, Map<String, Integer> termAndSumMap) throws SQLException {
        Db use = Db.use();
        int onePageNum = 1000;
        Set<String> early2020 = getEarly2020();
        Set<String> early2021 = getEarly2021();
        Set<String> early2023 = getEarly2023();
        Set<String> early2024 = getEarly2024();
        int tableMax = use.queryNumber("select max(" + autoId + ") from " + summaryTable).intValue();
        if (tableMax < 1) {
            return;
        }
        Db.use().execute("truncate table " + pmDataAnalysisTable);
        Map<String, String> infoAndUnitMap = getInfoAndUnitMap(standardUnitSumTable, "id", "info`,`unit");
        Collection<Integer> tableList = new ArrayList<>();
        NumberUtil.appendRange(0, tableMax + onePageNum, onePageNum, tableList);
        int docNumber = aidAndKeyWordMap.size();
        tableList.parallelStream().forEach(startId -> {
            try {
                List<Entity> tableData = use.query("select `" + autoId + "`,`" + field + "` from " + summaryTable + " where " + autoId + " between " + startId + " and " + (startId + onePageNum - 1) + limit);
                for (Entity entry : tableData) {
                    Integer id = entry.getInt(autoId);
                    if (null != id && id > 0) {

                        int pmId = Utils.getIntField(entry, "pmId");
                        int aId = Utils.getIntField(entry, "aid");
                        String ut = Utils.getStrField(entry, "ut").toUpperCase();
                        // 从老鲍的合并表中获取ut与pmId的映射，如果表中pmId为空，则使用映射的pmId值。
                        if (utMapPmIdMap.containsKey(ut) && pmId < 1) {
                            pmId = utMapPmIdMap.get(ut);
                        }

                        List<String> reList = new ArrayList<>();
                        if (aidAndReferenceMap.containsKey(aId)) {
                            byte[] bytes = aidAndReferenceMap.get(aId);
                            List<Integer> referencesList = strToList(ZipUtil.unGzip(bytes, "utf-8"));
                            for (Integer aid : referencesList) {
                                if (aidAndReferencesMap.containsKey(aid)) {
                                    reList.add(aidAndReferencesMap.get(aid));
                                }
                            }
                        }

                        List<Integer> citationsList = new ArrayList<>();
                        List<En.TiAuJ9> tiAuJ9List = new ArrayList<>();
                        if (aidAndCitationMap.containsKey(aId)) {
                            byte[] bytes = aidAndCitationMap.get(aId);
                            citationsList = strToList(ZipUtil.unGzip(bytes, "utf-8"));
                            for (Integer aid : citationsList) {
                                if (aidAndTiAuJ9Map.containsKey(aid)) {
                                    String str = ZipUtil.unGzip(aidAndTiAuJ9Map.get(aid), "utf-8");
                                    En.TiAuJ9 tiAuJ9 = JSONObject.parseObject(str, En.TiAuJ9.class);
                                    tiAuJ9List.add(tiAuJ9);
                                }
                            }
                        }

                        String c1Au = Utils.getStrField(entry, "c1_au");
                        List<En.C1Auth> c1List = c1Au.contains("sort") ? JSONObject.parseArray(c1Au, En.C1Auth.class) : new ArrayList<>();
                        Set<String> univOrHospSet = new HashSet<>();
                        List<String> countryList = new ArrayList<>();
                        for (En.C1Auth c1Auth : c1List) {
                            String country = c1Auth.getCountry();
                            if (country.length() > 1) {
                                countryList.add(country);
                            }
                            String unit = CheckUnit.listToUnit(new ArrayList<>(c1Auth.getUnitSet()));
                            if (unit.length() > 3) {
                                univOrHospSet.add(unit);
                            }
                        }

                        int year = Utils.getIntField(entry, "year");
                        List<En.C1Auth> c1Auths = new ArrayList<>();
                        // 如果pmId有ad相关信息，且有ut号，可以把pm的ad信息给追加到c1中，追加的数据，是以"ĺļ"为分隔点，之前的为原来的c1，之后的追加的pm的ad信息，这里采用的阈值是0.35。如果某ad信息只有pm有，而ut没有，则作者为"[Anonymous]"开头，前端显示时应给予屏蔽。
                        if (pmIdAndAdListMap.containsKey(pmId) && !ut.isEmpty()) {
                            String s = ZipUtil.unGzip(pmIdAndAdListMap.get(pmId), "utf-8");
                            List<String> pmAd = JSONObject.parseArray(s, String.class);
                            List<String> c1Ad = getC1Ad(c1Au);
                            Map<String, List<String>> bestAdMap = bestAd(pmAd, c1Ad);
                            c1Auths = upDataC1Au(c1Au, bestAdMap);
                        }
                        // 这个是不管是否正确的给予追加上，把所有的ad都给更新到"pm_ad"字段中。
                        String ads = pmIdAndAdListMap.containsKey(pmId) ? ZipUtil.unGzip(pmIdAndAdListMap.get(pmId), "utf-8") : "";

                        // 因为小葛的xml数据，没有解析出来图片章节"Book Chapter"类型，所以给添加上，如果后续小葛的数据中有了，可以不再加这行代码。
                        List<String> ptList = Utils.getFieldList(entry, "pt");
                        if (bookChapterSet.contains(pmId) && !ptList.contains("Book Chapter")) {
                            ptList.add("Book Chapter");
                        }

                        // 因发现基金字段有空值的现象，没有去七张表中找，直接在这里给予过滤，只保留有值的fList。
                        String funds = Utils.getStrField(entry, "fund_list");
                        List<String> fList = new ArrayList<>();
                        if (funds.contains("\"]")) {
                            List<String> fundList = JSONObject.parseArray(funds, String.class);
                            for (String fund : fundList) {
                                if (fund.length() > 1) {
                                    fList.add(fund);
                                }
                            }
                        }

                        // 生成小葛pm数据分析所需要使用的基金和单位的基础数据。
                        if (pmId > 0 && (!univOrHospSet.isEmpty() || !fList.isEmpty())) {
                            try {
                                Db.use().insert(Entity.create(pmDataAnalysisTable).set("pmId", pmId).set("fund_list", JSONUtil.toJsonStr(fList)).set("unit_list", JSONUtil.toJsonStr(univOrHospSet)));
                            } catch (SQLException e) {
                                e.printStackTrace();
                            }
                        }

                        // 对于是国家科学自然基金的论文，把作者加单位给单独抽出来，用于前端页面的聚类显示。
                        Set<En.FundAuth> authAndUnitSet = new HashSet<>();
                        String f = funds.toLowerCase();
                        if (year > 2017 && funds.contains("\"]") && (pmId > 0 || !ut.isEmpty()) && (f.contains("china ") || f.contains(" china") || f.contains("(nsfc)") || funds.contains("NSFC"))) {
                            List<String> fundList = JSONObject.parseArray(funds, String.class);
                            boolean bool = false;
                            // 标准的国家自然科学基金有2528357条，下面的一行代码可以减少大量的循环。
                            if (f.contains("national natural science foundation of china (nsfc)")) {
                                bool = true;
                            } else {
                                for (String fund : fundList) {
                                    if (isNsfc(fund)) {
                                        bool = true;
                                        break;
                                    }
                                }
                            }
                            if (bool) {
                                for (En.C1Auth c1Auth : c1List) {
                                    Set<String> unitSet = c1Auth.getUnitSet();
                                    String unit = CheckUnit.listToUnit(new ArrayList<>(unitSet));
                                    if (infoAndUnitMap.containsKey(unit)) {
                                        String hosp = infoAndUnitMap.get(unit);
                                        if (hosp.startsWith("中国_") && (hosp.contains("医院") || hosp.contains("妇幼保健院") || hosp.contains("精神卫生中心") || hosp.contains("疾病预防控制中心") || hosp.contains("妇婴保健院") ||
                                                hosp.contains("公共卫生临床中心") || hosp.contains("儿童医学中心") || hosp.contains("特色医学中心") || hosp.contains("儿童医疗中心") || hosp.contains("皮肤病防治所") || hosp.contains("牙科医疗中心") ||
                                                hosp.contains("临床医疗中心") || hosp.contains("海慈医疗集团") || hosp.contains("精神病院"))) {
                                            Set<String> deptSet = c1Auth.getDeptSet();
                                            List<String> deptList = new ArrayList<>();
                                            for (String dept : deptSet) {
                                                if (dept.startsWith("[hosp]")) {
                                                    deptList.add(dept.substring("[hosp]".length()));
                                                }
                                            }
                                            En.FundAuth fundAuth = new En.FundAuth();
                                            fundAuth.setAuthList(c1Auth.getAuthList());
                                            fundAuth.setDeptList(deptList);
                                            fundAuth.setUnit(hosp.substring("中国_".length()));
                                            authAndUnitSet.add(fundAuth);
                                        }
                                    }
                                }
                            }
                        }

                        String journal = Utils.getStrField(entry, "journal").toUpperCase();
                        String ncbiJournal = Utils.getStrField(entry, "ncbi_journal").toUpperCase();
                        String nlmId = Utils.getStrField(entry, "nlmId").toUpperCase();
                        int isEarly = 0;
                        // 预警期刊是按年来的，该年的预警期刊，该年发表的论文，才认定为预警。
                        if (year == 2020 && (early2020.contains(journal) || early2020.contains(ncbiJournal) || early2020.contains(nlmId))) {
                            isEarly = 1;
                        } else if (year == 2021 && (early2021.contains(journal) || early2021.contains(ncbiJournal) || early2021.contains(nlmId))) {
                            isEarly = 1;
                        } else if (year == 2023 && (early2023.contains(journal) || early2023.contains(ncbiJournal) || early2023.contains(nlmId))) {
                            isEarly = 1;
                        } else if (year == 2024 && (early2024.contains(journal) || early2024.contains(ncbiJournal) || early2024.contains(nlmId))) {
                            isEarly = 1;
                        }
                        String di = Utils.getStrField(entry, "doi");
                        String webSite = "";
                        if (doiAndLocationMap.containsKey(di)) {
                            webSite = doiAndLocationMap.get(di).toLowerCase().replaceAll("(?i)https:///", "").replaceAll("(?i)https://", "").replaceAll("(?i)http:///", "")
                                    .replaceAll("(?i)http://", "").replaceAll("(?i)www3.", "").replaceAll("(?i)www2.", "").replaceAll("(?i)www1.", "")
                                    .replaceAll("(?i)www.", "").replace("?", "/").replace("=", "/").replace(":", "/");
                            webSite = (webSite.indexOf("/") > 0) ? webSite.substring(0, webSite.indexOf("/")) : webSite;
                        }

                        List<List<String>> tiListList = new ArrayList<>();
                        List<List<String>> abListList = new ArrayList<>();
                        List<List<String>> kwListList = new ArrayList<>();
                        Map<String, Double> mapSortValue = new LinkedHashMap<>();
                        String str = "";
                        if (aidAndKeyWordMap.containsKey(aId)) {
                            str = ZipUtil.unGzip(aidAndKeyWordMap.get(aId), "utf-8");
                            En.KeyWords keyWords = JSONObject.parseObject(str, En.KeyWords.class);
                            tiListList = keyWords.getTiListList();
                            abListList = keyWords.getAbListList();
                            kwListList = keyWords.getKwListList();
                            List<List<String>> abbListList = keyWords.getAbbListList();
                            Map<String, String> mergeMap = new HashMap<>();
                            Map<String, String> meshMap = new HashMap<>();
                            List<String> allTermList = new ArrayList<>();
                            addAllTermList(mergeMap, meshMap, allTermList, 5, tiListList);
                            addAllTermList(mergeMap, meshMap, allTermList, 4, kwListList);
                            addAllTermList(mergeMap, meshMap, allTermList, 4, abbListList);
                            addAllTermList(mergeMap, meshMap, allTermList, 1, abListList);
                            // 在每篇论文中，计算每个主题出现的频次，出现多少算多少，因此，可能是大于1次的。
                            Map<String, Integer> termAndNumberMap = new HashMap<>();
                            for (String term : allTermList) {
                                termAndNumberMap.compute(term, (key, value) -> value != null ? (value + 1) : 1);
                            }
                            int totalTerms = allTermList.size();
                            Map<String, Double> termAndScoreMap = new LinkedHashMap<>();
                            Set<String> allTermSet = new HashSet<>(allTermList);
                            for (String term : allTermSet) {
                                // 计算TF （术语 t 在文档中出现的次数） / （文档中的术语总数）
                                double tf = (double) termAndNumberMap.getOrDefault(term, 0) / totalTerms;
                                // 计算IDF 某一特定词语的IDF，可以由总文件数目除以包含该词语的文件的数目，再将得到的商取对数得到。
                                double idf = Math.log((double) docNumber / (termAndSumMap.getOrDefault(term, 0) + 1e-12));
                                // 计算TF-IDF
                                termAndScoreMap.put(term, tf * idf);
                            }
                            // 合并mergeMap，键为全写，值为缩写，合并时把全写和缩写的值相累加。
                            for (Map.Entry<String, String> map : mergeMap.entrySet()) {
                                String full = map.getKey();
                                String abbr = map.getValue();
                                if (termAndScoreMap.containsKey(full) && termAndScoreMap.containsKey(abbr)) {
                                    double score = termAndScoreMap.get(full) + termAndScoreMap.get(abbr);
                                    termAndScoreMap.remove(abbr);
                                    termAndScoreMap.put(full, score);
                                }
                            }
                            // 对于mesh，则取最大的那个值。
                            for (Map.Entry<String, String> map : meshMap.entrySet()) {
                                String original = map.getKey();
                                String mesh = map.getValue();
                                if (termAndScoreMap.containsKey(original) && termAndScoreMap.containsKey(mesh)) {
                                    double max = Math.max(termAndScoreMap.get(original), termAndScoreMap.get(mesh));
                                    termAndScoreMap.remove(original);
                                    termAndScoreMap.put(mesh, max);
                                }
                            }
                            Map<String, Double> sortMap = Utils.mapSortValue(termAndScoreMap);
                            for (Map.Entry<String, Double> map : sortMap.entrySet()) {
                                mapSortValue.put(map.getKey(), Utils.takeFourDigits(map.getValue()));
                            }
                        }
                        try {
                            Db.use().insert(Entity.create(summaryFinalTable).set("key_word_all", str).set("early_warning", isEarly).set("is_china", countryList.contains("中国") ? 1 : 0).set("cn_country", JSONUtil.toJsonStr(countryList)).set("c1_au", JSONUtil.toJsonStr(c1Auths))
                                    .set("web_site", webSite).set("citations", JSONUtil.toJsonStr(tiAuJ9List)).set("pt", JSONUtil.toJsonStr(ptList)).set("fund_list", JSONUtil.toJsonStr(fList)).set("is_fund", authAndUnitSet.isEmpty() ? 0 : 1).set("fund_auth", JSONUtil.toJsonStr(authAndUnitSet))
                                    .set("times_cited", citationsList.size()).set("pm_ad", ads).set("pmId", pmId).set("references", CollUtil.join(reList, "; ")).set("ti_key", JSONUtil.toJsonStr(tiListList)).set("kw_key", JSONUtil.toJsonStr(kwListList))
                                    .set("ab_key", JSONUtil.toJsonStr(abListList)).set("key_word_list", JSONUtil.toJsonStr(mapSortValue)).set("citation_quota", entry.getDouble("citation_quota")).set("reference_quota", entry.getDouble("reference_quota")).set("jcr_if", entry.getDouble("jcr_if"))
                                    .set("cite_score", entry.getDouble("cite_score")).set("cs_sjr", entry.getDouble("cs_sjr")).set("cscd_id", entry.getInt("cscd_id")).set("ncbi_id", entry.getInt("ncbi_id")).set("yiigle_id", entry.getInt("yiigle_id"))
                                    .set("wos_id", entry.getInt("wos_id")).set("weipu_id", entry.getInt("weipu_id")).set("wanfang_id", entry.getInt("wanfang_id")).set("cnki_id", entry.getInt("cnki_id")).set("aid", entry.getInt("aid")).set("zky_dalei", entry.getInt("zky_dalei"))
                                    .set("year", entry.getInt("year")).set("jid", entry.getInt("jid")).set("is_cns", entry.getInt("is_cns")).set("jcr_quartile", entry.getInt("jcr_quartile")).set("is_retraction", entry.getInt("is_retraction")).set("is_beida", entry.getInt("is_beida"))
                                    .set("is_cs", entry.getInt("is_cs")).set("is_cscd", entry.getInt("is_cscd")).set("is_jcr", entry.getInt("is_jcr")).set("is_pubmed", entry.getInt("is_pubmed")).set("is_tjy", entry.getInt("is_tjy")).set("is_yiigle", entry.getInt("is_yiigle"))
                                    .set("is_zky", entry.getInt("is_zky")).set("is_core", entry.getInt("is_core")).set("is_china_hosp", entry.getInt("is_china_hosp")).set("is_guidelines", entry.getInt("is_guidelines")).set("is_pubmed_article", entry.getInt("is_pubmed_article"))
                                    .set("is_6year", entry.getInt("is_6year")).set("abb_word", entry.getStr("abb_word")).set("keyword", entry.getStr("keyword")).set("ab", entry.getStr("ab")).set("discussion", entry.getStr("discussion")).set("results", entry.getStr("results"))
                                    .set("methods", entry.getStr("methods")).set("background", entry.getStr("background")).set("rp_au", entry.getStr("rp_au")).set("cite", entry.getStr("cite")).set("cn_ab", entry.getStr("cn_ab")).set("cn_kw", entry.getStr("cn_kw"))
                                    .set("drug_gene_pathway_disease", entry.getStr("drug_gene_pathway_disease")).set("auth", entry.getStr("auth")).set("info", entry.getStr("info")).set("cn_discussion", entry.getStr("cn_discussion")).set("cn_results", entry.getStr("cn_results"))
                                    .set("cn_methods", entry.getStr("cn_methods")).set("cn_background", entry.getStr("cn_background")).set("ncbi_subject", entry.getStr("ncbi_subject")).set("ncbi_mesh", entry.getStr("ncbi_mesh")).set("journal", entry.getStr("journal"))
                                    .set("title", entry.getStr("title")).set("mesh", entry.getStr("mesh")).set("cn_ti", entry.getStr("cn_ti")).set("cn_type", entry.getStr("cn_type")).set("doi", entry.getStr("doi")).set("zky_subject", entry.getStr("zky_subject"))
                                    .set("mail_list", entry.getStr("mail_list")).set("subject", entry.getStr("subject")).set("kw_plus", entry.getStr("kw_plus")).set("language", entry.getStr("language")).set("ncbi_journal", entry.getStr("ncbi_journal"))
                                    .set("ut", entry.getStr("ut")).set("nlmId", entry.getStr("nlmId")).set("url", entry.getStr("url")));
                        } catch (SQLException e) {
                            e.printStackTrace();
                        }
                    }
                }
            } catch (SQLException e) {
                e.printStackTrace();
            }
        });
    }

    // type分别为标题、关键词、全缩写、摘要四种，值分别为3、3、3、1，值为3表示出现一次按3次计算。
    public static void addAllTermList(Map<String, String> mergeMap, Map<String, String> meshMap, List<String> allTermList, int type, List<List<String>> tiListList) {
        for (List<String> termList : tiListList) {
            if (termList.size() == 3) {
                for (int i = 0; i < type; i++) {
                    allTermList.add(termList.get(0));
                    allTermList.add(termList.get(1));
                }
                meshMap.put(termList.get(0), termList.get(1));
            } else if (termList.size() == 2) {
                for (int i = 0; i < type; i++) {
                    allTermList.add(termList.get(0));
                    allTermList.add(termList.get(1));
                }
                mergeMap.put(termList.get(0), termList.get(1));
            } else if (termList.size() == 1) {
                for (int i = 0; i < type; i++) {
                    allTermList.add(termList.getFirst());
                }
            }
        }
    }

    public static Map<String, String> getInfoAndUnitMap(String standardUnitSumTable, String autoId, String field) throws SQLException {
        int onePageNum = 1000;
        List<Entity> tableNum = Db.use().query("select count(1) as count from " + standardUnitSumTable);
        Map<String, String> infoAndUnitMap = new HashMap<>();
        if (tableNum.getFirst().getInt("count") > 0) {
            int tableMax = Db.use().queryNumber("select max(" + autoId + ") from " + standardUnitSumTable).intValue();
            Collection<Integer> tableList = new ArrayList<>();
            NumberUtil.appendRange(0, tableMax + onePageNum, onePageNum, tableList);
            for (Integer startId : tableList) {
                List<Entity> tableData = Db.use().query("select `" + autoId + "`,`" + field + "` from " + standardUnitSumTable + " where " + autoId + " between " + startId + " and " + (startId + onePageNum - 1));
                for (Entity entry : tableData) {
                    String info = Utils.getStrField(entry, "info");
                    String unit = Utils.getStrField(entry, "unit");
                    if (info.length() > 3 && unit.length() > 6) {
                        infoAndUnitMap.put(info, unit);
                    }
                }
            }
        }
        return infoAndUnitMap;
    }

    public static boolean isNsfc(String info) {
        String l = info.toLowerCase().trim();
        if (l.contains("china ") || l.contains(" china") || info.contains("NSFC") || l.contains("(nsfc)")) {
            if (l.contains("province") || l.contains("research") || l.contains("provincial") || l.contains("technology") || l.contains("social") || l.contains("education") || l.contains("scholarship") ||
                    l.contains("973") || l.contains("111") || l.contains("863") || l.contains("postdoctoral") || l.contains("university") || l.contains("universities") || l.contains("key r&d")) {
                return false;
            }
            if (l.contains("(nsfc)") || Pattern.compile("nati[a-z ]+natu[a-z ]+scie[a-z ]+fund[a-z ]+|nati[a-z ]+natu[a-z ]+scie[a-z ]+foun[a-z ]+").matcher(l).find() || info.contains("NSF of China") || info.contains("NSFC")) {
                return true;
            }
            return l.contains("national science foundation of china") || l.contains("national natural science foun-dation of china") || l.contains("national natural sci-ence foundation of china") || l.contains("nature science foundation of china") ||
                    l.contains("national natural sci- ence foundation of china") || l.contains("national natural foundation of china") || l.contains("na-tional natural science foundation of china") || l.contains("national nat-ural science foundation of china") ||
                    l.contains("national natural science of china") || l.contains("na- tional natural science foundation of china") || l.contains("national scientific foundation of china") || l.contains("natural science foundations of china") ||
                    l.contains("china natural science foundation") || l.contains("national nat- ural science foundation of china") || l.contains("national science fund of china") || l.contains("natural sciences foundation of china") ||
                    l.contains("china national science foundation") || l.contains("nsf china") || l.contains("china nsf") || l.contains("natural science fund of china") || l.contains("national natu-ral science foundation of china") ||
                    l.contains("national sciences foundation of china") || l.contains("natural national science foundation of china") || l.contains("national foundation of china") || l.contains("national science funds of china") ||
                    l.contains("national science foundation china") || "science foundation of china".equals(l) || "national science foundation (nsf)".equals(l);
        }
        return false;
    }

    public static List<Integer> strToList(String str) {
        List<Integer> list = new ArrayList<>();
        if (null != str && !str.isEmpty()) {
            String[] split = str.split(",");
            for (String s : split) {
                if (NumberUtil.isInteger(s.trim())) {
                    int aid = Integer.parseInt(s.trim());
                    if (aid > 0) {
                        list.add(aid);
                    }
                }
            }
        }
        return list;
    }

    public static Map<Integer, String> getAidAndReferencesMap(String summaryTable, String autoId, String field, String limit, Map<Integer, byte[]> aidAndTiAuJ9Map) throws SQLException {
        Db use = Db.use();
        int onePageNum = 1000;
        List<Entity> tableNum = use.query("select count(1) as count from " + summaryTable);
        Map<Integer, String> aidAndReferencesMap = new ConcurrentHashMap<>();
        if (tableNum.getFirst().getInt("count") > 0) {
            int tableMax = use.queryNumber("select max(" + autoId + ") from " + summaryTable).intValue();
            Collection<Integer> tableList = new ArrayList<>();
            NumberUtil.appendRange(0, tableMax + onePageNum, onePageNum, tableList);
            tableList.parallelStream().forEach(startId -> {
                try {
                    List<Entity> tableData = use.query("select `" + autoId + "`,`" + field + "` from " + summaryTable + " where " + autoId + " between " + startId + " and " + (startId + onePageNum - 1) + limit);
                    for (Entity entry : tableData) {
                        Integer aid = entry.getInt("aid");
                        if (null != aid && aid > 0) {
                            int year = Utils.getIntField(entry, "year");
                            List<String> authList = Utils.getFieldList(entry, "auth");
                            String journal = Utils.getStrField(entry, "journal");
                            String auth = !authList.isEmpty() ? authList.get(0) : "[Anonymous]";
                            String info = auth + ", " + year + ", " + journal + ", doi 10.1086/" + aid;
                            String title = Utils.getStrField(entry, "title");
                            String cnTitle = Utils.getStrField(entry, "cn_ti");
                            En.TiAuJ9 tiAuJ9 = new En.TiAuJ9();
                            tiAuJ9.setTi(title.isEmpty() ? cnTitle : title);
                            tiAuJ9.setAu(CollUtil.join((authList.size() > 5 ? authList.subList(0, 5) : authList), "; "));
                            tiAuJ9.setSo(Utils.getStrField(entry, "info"));
                            tiAuJ9.setPmId(Utils.getIntField(entry, "pmId"));
                            tiAuJ9.setAId(aid);
                            byte[] zip = ZipUtil.gzip(JSONUtil.toJsonStr(tiAuJ9), "utf-8");
                            aidAndTiAuJ9Map.put(aid, zip);
                            aidAndReferencesMap.put(aid, info.replace(", , ", ", ").replace("; ", ", ").trim());
                        }
                    }
                } catch (SQLException e) {
                    e.printStackTrace();
                }
            });
        }
        return aidAndReferencesMap;
    }

    public static Set<Integer> getBookChapterSet(String ncTextTable, String autoId, String field, String limit) throws SQLException {
        Db use = Db.use();
        int onePageNum = 1000;
        List<Entity> tableNum = use.query("select count(1) as count from " + ncTextTable);
        Set<Integer> bookChapterSet = new ConcurrentHashSet<>();
        if (tableNum.getFirst().getInt("count") > 0) {
            int tableMax = use.queryNumber("select max(" + autoId + ") from " + ncTextTable).intValue();
            Collection<Integer> tableList = new ArrayList<>();
            NumberUtil.appendRange(0, tableMax + onePageNum, onePageNum, tableList);
            tableList.parallelStream().forEach(startId -> {
                try {
                    List<Entity> tableData = use.query("select `" + autoId + "`,`" + field + "` from " + ncTextTable + " where " + autoId + " between " + startId + " and " + (startId + onePageNum - 1) + limit);
                    for (Entity entry : tableData) {
                        Integer pmId = entry.getInt("pmid");
                        String pts = entry.getStr("ptList");
                        if (null != pmId && pmId > 0 && null != pts && pts.toLowerCase().contains("book chapter")) {
                            bookChapterSet.add(pmId);
                        }
                    }
                } catch (SQLException e) {
                    e.printStackTrace();
                }
            });
        }
        return bookChapterSet;
    }

    public static void getDoiAndLocationMap(String doiTable, String autoId, String field, Map<String, String> doiAndLocationMap, String limit) throws SQLException {
        Db use = Db.use();
        int onePageNum = 1000;
        List<Entity> tableNum = use.query("select count(1) as count from " + doiTable);
        if (tableNum.getFirst().getInt("count") > 0) {
            int tableMax = use.queryNumber("select max(" + autoId + ") from " + doiTable).intValue();
            Collection<Integer> tableList = new ArrayList<>();
            NumberUtil.appendRange(0, tableMax + onePageNum, onePageNum, tableList);
            tableList.parallelStream().forEach(startId -> {
                try {
                    List<Entity> tableData = use.query("select `" + autoId + "`,`" + field + "` from " + doiTable + " where " + autoId + " between " + startId + " and " + (startId + onePageNum - 1) + limit);
                    for (Entity entry : tableData) {
                        String doi = entry.getStr("doi");
                        String location = entry.getStr("location");
                        if (null != doi && null != location && doi.contains("10.") && (location.toLowerCase().contains("http") || location.toLowerCase().contains("www"))) {
                            doiAndLocationMap.put(doi.toLowerCase().trim(), location);
                        }
                    }
                } catch (SQLException e) {
                    e.printStackTrace();
                }
            });
        }
    }

    public static Map<Integer, byte[]> getPmIdAndAdListMap(String pmArticleTable, String autoId, String field, String limit) throws SQLException {
        List<Entity> tableNum = Db.use().query("select count(1) as count from " + pmArticleTable);
        int onePageNum = 1000;
        Map<Integer, byte[]> pmIdAndAdListMap = new ConcurrentHashMap<>();
        if (tableNum.getFirst().getInt("count") > 0) {
            int tableMax = Db.use().queryNumber("select max(" + autoId + ") from " + pmArticleTable).intValue();
            Collection<Integer> tableList = new ArrayList<>();
            NumberUtil.appendRange(0, tableMax + onePageNum, onePageNum, tableList);
            tableList.parallelStream().forEach(startId -> {
                try {
                    List<Entity> tableData = Db.use().query("select `" + autoId + "`,`" + field + "` from " + pmArticleTable + " where " + autoId + " between " + startId + " and " + (startId + onePageNum - 1) + limit);
                    for (Entity entry : tableData) {
                        Integer pmId = entry.getInt("pmid");
                        String authorExtend = entry.getStr("author_extend");
                        if (null != pmId && pmId > 0 && null != authorExtend && authorExtend.contains("Affiliation")) {
                            JSONArray objects = JSONUtil.parseArray(authorExtend);
                            List<String> adList = new ArrayList<>();
                            for (Object object : objects) {
                                JSON parse = JSONUtil.parse(object);
                                String affiliation = parse.getByPath("Affiliation").toString();
                                List<String> infoList = com.alibaba.fastjson.JSON.parseArray(affiliation, String.class);
                                for (String s : infoList) {
                                    if (!adList.contains(s) && s.length() > 8) {
                                        adList.add(s);
                                    }
                                }
                            }
                            if (!adList.isEmpty()) {
                                byte[] zip = ZipUtil.gzip(JSONUtil.toJsonStr(adList), "utf-8");
                                pmIdAndAdListMap.put(pmId, zip);
                            }
                        }
                    }
                } catch (SQLException e) {
                    e.printStackTrace();
                }
            });
        }
        return pmIdAndAdListMap;
    }

    public static Map<String, Integer> getUtMapPmIdMap(String basicCoreTable, String autoId, String field, String limit) throws SQLException {
        Db use = Db.use();
        int onePageNum = 1000;
        List<Entity> tableNum = use.query("select count(1) as count from " + basicCoreTable);
        Map<String, Integer> utMapPmIdMap = new ConcurrentHashMap<>();
        if (tableNum.getFirst().getInt("count") > 0) {
            int tableMax = use.queryNumber("select max(" + autoId + ") from " + basicCoreTable).intValue();
            Collection<Integer> tableList = new ArrayList<>();
            NumberUtil.appendRange(0, tableMax + onePageNum, onePageNum, tableList);
            tableList.parallelStream().forEach(startId -> {
                try {
                    List<Entity> tableData = use.query("select `" + autoId + "`,`" + field + "` from " + basicCoreTable + " where " + autoId + " between " + startId + " and " + (startId + onePageNum - 1) + limit);
                    for (Entity entry : tableData) {
                        String pm = entry.getStr("pm");
                        String ut = entry.getStr("wos_ut");
                        // 只考虑一个的情况，如果有多个ut或多个pm，则完全不考虑。
                        if (null != pm && NumberUtil.isInteger(pm) && null != ut && ut.length() == 15) {
                            utMapPmIdMap.put(ut.toUpperCase(), Integer.parseInt(pm));
                        }
                    }
                } catch (SQLException e) {
                    e.printStackTrace();
                }
            });
        }
        return utMapPmIdMap;
    }

    public static Map<Integer, byte[]> getAidAndCitationMap(String tableName, String autoId, String field, String type, String limit) throws SQLException {
        List<Entity> tableNum = Db.use().query("select count(1) as count from " + tableName);
        int onePageNum = 1000;
        Map<Integer, byte[]> aidAndByteMap = new ConcurrentHashMap<>();
        if (tableNum.getFirst().getInt("count") > 0) {
            int tableMax = Db.use().queryNumber("select max(" + autoId + ") from " + tableName).intValue();
            Collection<Integer> tableList = new ArrayList<>();
            NumberUtil.appendRange(0, tableMax + onePageNum, onePageNum, tableList);
            tableList.parallelStream().forEach(startId -> {
                try {
                    List<Entity> tableData = Db.use().query("select `" + autoId + "`,`" + field + "` from " + tableName + " where " + autoId + " between " + startId + " and " + (startId + onePageNum - 1) + limit);
                    for (Entity entry : tableData) {
                        int aid = Utils.getIntField(entry, "pid");
                        if ("citation".equals(type)) {
                            String citation = Utils.getStrField(entry, "citation_id");
                            if (aid > 0 && !citation.isEmpty()) {
                                byte[] zip = ZipUtil.gzip(citation, "utf-8");
                                aidAndByteMap.put(aid, zip);
                            }
                        }
                        if ("reference".equals(type)) {
                            String reference = Utils.getStrField(entry, "reference_id");
                            if (aid > 0 && !reference.isEmpty()) {
                                byte[] zip = ZipUtil.gzip(reference, "utf-8");
                                aidAndByteMap.put(aid, zip);
                            }
                        }
                    }
                } catch (SQLException e) {
                    e.printStackTrace();
                }
            });
        }
        return aidAndByteMap;
    }

    public static Map<String, byte[]> getTitleTransMap(String titleTransTable, String autoId, String field, String limit) throws SQLException {
        List<Entity> tableNum = Db.use().query("select count(1) as count from " + titleTransTable);
        int onePageNum = 1000;
        Map<String, byte[]> titleTransMap = new ConcurrentHashMap<>();
        if (tableNum.getFirst().getInt("count") > 0) {
            int tableMax = Db.use().queryNumber("select max(" + autoId + ") from " + titleTransTable).intValue();
            Collection<Integer> tableList = new ArrayList<>();
            NumberUtil.appendRange(0, tableMax + onePageNum, onePageNum, tableList);
            tableList.parallelStream().forEach(startId -> {
                try {
                    List<Entity> tableData = Db.use().query("select `" + autoId + "`,`" + field + "` from " + titleTransTable + " where " + autoId + " between " + startId + " and " + (startId + onePageNum - 1) + limit);
                    for (Entity entry : tableData) {
                        String original = Utils.getStrField(entry, "original");
                        String trans = Utils.getStrField(entry, "trans");
                        if (original.length() > 3 && trans.length() > 3) {
                            String md5Name = SecureUtil.md5(original).toLowerCase();
                            titleTransMap.put(md5Name, ZipUtil.gzip(trans, "utf-8"));
                        }
                    }
                } catch (SQLException e) {
                    e.printStackTrace();
                }
            });
        }
        return titleTransMap;
    }

    // 全部分翻译，不考虑年份及其它。但存在原始的数据中，英文的标题是中文，中文的是英文的现象，本方法不考虑，只管直接更新。入ES检索或展示时需要加入区分。
    public static void transTitle(String summaryFinalTable, String autoId, String field, int min, int max, Map<String, byte[]> titleTransMap, String titleTransTable) throws SQLException {
        SimpleDateFormat format = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
        int onePageNum = 10000;
        int tableMax = Db.use().queryNumber("select max(" + autoId + ") from " + summaryFinalTable).intValue();
        if (tableMax > 0) {
            Collection<Integer> tableList = new ArrayList<>();
            NumberUtil.appendRange(0, tableMax + onePageNum, onePageNum, tableList);
            for (Integer startId : tableList) {
                List<Entity> tableData = Db.use().query("select `" + autoId + "`,`" + field + "` from " + summaryFinalTable + " where " + autoId + " between " + startId + " and " + (startId + onePageNum - 1));
                System.out.println("\033[31;1m" + format.format(new Date()) + " 标题翻译：" + startId + " 表名：" + summaryFinalTable + Utils.printMemoryStr() + "\033[0m");
                for (Entity entry : tableData) {
                    int id = Utils.getIntField(entry, autoId);
                    int pmId = Utils.getIntField(entry, "pmId");
                    String ut = Utils.getStrField(entry, "ut");
                    String title = Utils.getStrField(entry, "title");
                    String cnTi = Utils.getStrField(entry, "cn_ti");
                    // 第一种情况，英文标题为空，中文有标题。
                    if (title.length() < 4 && cnTi.length() > 1) {
                        String md5Name = SecureUtil.md5(cnTi).toLowerCase();
                        boolean isCn = Utils.isContainChinese(cnTi);
                        if (isCn) {
                            String trans = titleTransMap.containsKey(md5Name) ? ZipUtil.unGzip(titleTransMap.get(md5Name), "utf-8") : ("ĺļ" + TransApi.youDaoCnToEnTrans(cnTi));
                            updateInsertTable(summaryFinalTable, ut, pmId, autoId, "title", 1, id, cnTi, trans, titleTransTable, "", !titleTransMap.containsKey(md5Name));
                        } else {
                            // 这种情况是中文标题内的不是中文，而是英文，而英文中的又没有，也就是说放反了。
                            String trans = titleTransMap.containsKey(md5Name) ? ZipUtil.unGzip(titleTransMap.get(md5Name), "utf-8") : ("ĺļ" + TransApi.youDaoEnToCnTrans(cnTi));
                            updateInsertTable(summaryFinalTable, ut, pmId, autoId, "title", 2, id, cnTi, trans, titleTransTable, "", !titleTransMap.containsKey(md5Name));
                        }
                    }
                    // 第二种情况，pm或ut有值，英文标题有值，中文标题无值，此类属于标准的pm或ut的数据，原始的数据是中文肯定是空的。
                    if ((pmId > 0 || ut.length() >= 15) && title.length() > 3 && cnTi.length() < 4) {
                        String trans = "";
                        String ti = title.trim();
                        String md5Name = SecureUtil.md5(title).toLowerCase();
                        if (titleTransMap.containsKey(md5Name)) {
                            trans = ZipUtil.unGzip(titleTransMap.get(md5Name), "utf-8");
                        } else {
                            ti = ti.startsWith("[") && ti.endsWith("].") ? ti.substring("[".length(), ti.length() - "].".length()).trim() : ti;
                            ti = ti.startsWith("[") && ti.endsWith("]") ? ti.substring("[".length(), ti.length() - "]".length()).trim() : ti;
                            ti = Pattern.compile("<[^>]*>").matcher(ti).find() ? Jsoup.parse(ti).text().trim() : ti;
                            String f = ti.startsWith("被撤回的出版物:") || ti.startsWith("RETRACTED:") ? "被撤回的出版物: " : "";
                            f = ti.startsWith("撤稿声明:") ? "撤稿声明: " : f;
                            ti = ti.startsWith("被撤回的出版物:") ? ti.substring("被撤回的出版物:".length()).trim() : ti;
                            ti = ti.startsWith("撤稿声明:") ? ti.substring("撤稿声明:".length()).trim() : ti;
                            ti = ti.startsWith("RETRACTED:") ? ti.substring("RETRACTED:".length()).trim() : ti;
                            trans = "ĺļ" + f + TransApi.youDaoEnToCnTrans(ti);
                        }
                        updateInsertTable(summaryFinalTable, ut, pmId, autoId, "cn_ti", 0, id, title, trans, titleTransTable, ti, !titleTransMap.containsKey(md5Name));
                    }
                    // 第三种情况，pm和ut都没有值，英文标题有值，中文标题无值，此类一般出现在CSCD中，且说是英文标题也不一定就是英文的。
                    if (pmId < 1 && ut.length() < 15 && title.length() > 3 && cnTi.length() < 2) {
                        String md5Name = SecureUtil.md5(title).toLowerCase();
                        boolean isEn = !Utils.isContainChinese(title);
                        if (isEn) {
                            String trans = titleTransMap.containsKey(md5Name) ? ZipUtil.unGzip(titleTransMap.get(md5Name), "utf-8") : ("ĺļ" + TransApi.youDaoEnToCnTrans(title));
                            updateInsertTable(summaryFinalTable, ut, pmId, autoId, "cn_ti", 3, id, title, trans, titleTransTable, "", !titleTransMap.containsKey(md5Name));
                        } else {
                            String trans = titleTransMap.containsKey(md5Name) ? ZipUtil.unGzip(titleTransMap.get(md5Name), "utf-8") : ("ĺļ" + TransApi.youDaoCnToEnTrans(title));
                            updateInsertTable(summaryFinalTable, ut, pmId, autoId, "cn_ti", 4, id, title, trans, titleTransTable, "", !titleTransMap.containsKey(md5Name));
                        }
                    }
                }
            }
        }
    }

    public static void updateInsertTable(String summaryTitleTable, String ut, int pmId, String autoId, String field, int type, int id, String original, String trans, String titleTransTable, String ti, boolean isNew) {
        if (trans.length() > 3) {
            try {
                Db.use().update(Entity.create().set(field, trans), Entity.create(summaryTitleTable).set(autoId, id));
                if (isNew) {
                    Db.use().insert(Entity.create(titleTransTable).set("ut", ut).set("pm", pmId).set("type", type).set("original", original).set("trans", trans).set("ti", ti));
                }
            } catch (SQLException e) {
                e.printStackTrace();
            }
        }
    }

    public static void insertReferenceTable(String citationTable, String autoId, String field, String referenceTable, String limit) throws SQLException {
        int onePageNum = 100000;
        int tableMax = Db.use().queryNumber("select max(" + autoId + ") from " + citationTable).intValue();
        Map<Integer, Set<Integer>> pidAndPidSetMap = new ConcurrentHashMap<>();
        if (tableMax > 0) {
            Collection<Integer> tableList = new ArrayList<>();
            NumberUtil.appendRange(0, tableMax + onePageNum, onePageNum, tableList);
            tableList.parallelStream().forEach(startId -> {
                try {
                    List<Entity> tableData = Db.use().query("select `" + autoId + "`,`" + field + "` from " + citationTable + " where " + autoId + " between " + startId + " and " + (startId + onePageNum - 1) + limit);
                    for (Entity entry : tableData) {
                        Integer pid = entry.getInt("pid");
                        String citationId = entry.getStr("citation_id");
                        if (null != pid && pid > 0 && null != citationId && !citationId.isEmpty()) {
                            String[] split = citationId.replace(" ", "").trim().split(",");
                            for (String ids : split) {
                                if (NumberUtil.isInteger(ids)) {
                                    putIntSetIntMap(pidAndPidSetMap, Integer.parseInt(ids), pid);
                                }
                            }
                        }
                    }
                } catch (SQLException e) {
                    throw new RuntimeException(e);
                }
            });
        }
        pidAndPidSetMap.keySet().parallelStream().forEach(pid -> {
            Set<Integer> idSet = pidAndPidSetMap.get(pid);
            String ids = CollUtil.join(idSet, ",");
            try {
                Db.use().insert(Entity.create(referenceTable).set("pid", pid).set("reference_total", idSet.size()).set("reference_id", ids));
            } catch (SQLException e) {
                e.printStackTrace();
            }
        });
        pidAndPidSetMap.clear();
    }

    public static void putIntSetIntMap(Map<Integer, Set<Integer>> finishAidListSetMap, int max, int min) {
        finishAidListSetMap.compute(max, (k, v) -> {
            if (v == null) {
                v = new ConcurrentHashSet<>();
            }
            v.add(min);
            return v;
        });
    }

    public static Set<String> getEarly2024() {
        Set<String> set = new HashSet<>();
        set.add("ENVIRON SCI POLLUT R");
        set.add("EXP THER MED");
        set.add("J CLIN MED");
        set.add("FRONTIERS IN ENERGY RESEARCH");
        set.add("JOURNAL OF PERSONALIZED MEDICINE");
        set.add("J BIOMATER TISSUE ENG");
        set.add("JOURNAL OF NANOELECTRONICS AND OPTOELECTRONICS");
        set.add("MULTIMED TOOLS APPL");
        set.add("PSYCHIATRIA DANUBINA");
        set.add("FUEL");
        set.add("SCIENCE OF ADVANCED MATERIALS");
        set.add("J SENS");
        set.add("MATH BIOSCI ENG");
        set.add("RADIOLOGIA MEDICA");
        set.add("EXPERIMENTAL AND THERAPEUTIC MEDICINE");
        set.add("MATER EXPRESS");
        set.add("CONNECT SCI");
        set.add("MULTIMEDIA TOOLS AND APPLICATIONS");
        set.add("RADIOL MED");
        set.add("DIAGNOSTICS");
        set.add("MATERIALS EXPRESS");
        set.add("J SENSORS");
        set.add("JOURNAL OF BIOMEDICAL NANOTECHNOLOGY");
        set.add("CONNECTION SCIENCE");
        set.add("CANCERS");
        set.add("FRONT ENERGY RES");
        set.add("ALTERNATIVE THERAPIES IN HEALTH AND MEDICINE");
        set.add("J BIOMATER TISS ENG");
        set.add("JOURNAL OF BIOMATERIALS AND TISSUE ENGINEERING");
        set.add("J NANOELECTRON OPTOELECTRON");
        set.add("PSYCHIATR DANUB");
        set.add("J NANOELECTRON OPTOE");
        set.add("CMES COMP MODEL ENG");
        set.add("ALTERN THER HEALTH M");
        set.add("CMES COMPUTER MODELING IN ENGINEERING & SCIENCES");
        set.add("J BIOBASED MATER BIOENERGY");
        set.add("JOURNAL OF CLINICAL MEDICINE");
        set.add("TROPICAL JOURNAL OF PHARMACEUTICAL RESEARCH");
        set.add("J PERS MED");
        set.add("PSYCHIAT DANUB");
        set.add("ENVIRONMENTAL SCIENCE AND POLLUTION RESEARCH");
        set.add("TROP J PHARM RES");
        set.add("ENVIRON SCI POLLUT RES");
        set.add("SCI ADV MATER");
        set.add("J BIOMED NANOTECHNOL");
        set.add("MATHEMATICAL BIOSCIENCES AND ENGINEERING");
        set.add("JOURNAL OF SENSORS");
        set.add("J BIOBASED MATER BIO");
        set.add("JOURNAL OF BIOBASED MATERIALS AND BIOENERGY");
        set.add("ALTERN THER HEALTH MED");
        set.add("CMES COMP MODEL ENG SCI");
        set.add("BIOENGINEERED");
        set.add("101526829");
        set.add("101658402");
        set.add("9441769");
        set.add("101556828");
        set.add("101606588");
        set.add("101602269");
        set.add("0177625");
        set.add("101581063");
        set.add("9012163");
        set.add("101555932");
        set.add("9424753");
        set.add("101291525");
        set.add("101591609");
        set.add("101230869");
        set.add("101703746");
        set.add("101598992");
        set.add("9502013");
        set.add("101511516");
        set.add("101531947");
        set.add("101638683");
        set.add("101197794");
        set.add("101214391");
        set.add("CMES-COMPUTER MODELING IN ENGINEERING & SCIENCES");
        set.add("0944-1344");
        set.add("0016-2361");
        set.add("0033-8362");
        set.add("2165-5979");
        set.add("0954-0091");
        set.add("1380-7501");
        set.add("0353-5053");
        set.add("1556-6560");
        set.add("2157-9083");
        set.add("1550-7033");
        set.add("1555-130X");
        set.add("1687-725X");
        set.add("2158-5849");
        set.add("1947-2935");
        set.add("1078-6791");
        set.add("1526-1492");
        set.add("1792-0981");
        set.add("2296-598X");
        set.add("1547-1063");
        set.add("1596-5996");
        set.add("2072-6694");
        set.add("2075-4418");
        set.add("1614-7499");
        set.add("1873-7153");
        set.add("2077-0383");
        set.add("2075-4426");
        set.add("1826-6983");
        set.add("2165-5987");
        set.add("1360-0494");
        set.add("1573-7721");
        set.add("1849-0867");
        set.add("1556-6579");
        set.add("2157-9091");
        set.add("1550-7041");
        set.add("1555-1318");
        set.add("1687-7268");
        set.add("2158-5857");
        set.add("1947-2943");
        set.add("1526-1506");
        set.add("1792-1015");
        set.add("1551-0018");
        set.add("CMES-COMP MODEL ENG");
        set.add("ENVIRON. SCI. POLLUT. RES.");
        set.add("J. CLIN. MED.");
        set.add("J. PERS. MED.");
        set.add("RADIOL. MED.");
        set.add("CONNECT. SCI.");
        set.add("MULTIMED. TOOLS APPL.");
        set.add("PSYCHIATR. DANUB.");
        set.add("J. BIOBASED MATER. BIOENERGY");
        set.add("J. BIOMATER. TISSUE ENG.");
        set.add("J. BIOMED. NANOTECHNOL.");
        set.add("J. NANOELECTRON. OPTOELECTRON.");
        set.add("J. SENS.");
        set.add("MATER. EXPRESS");
        set.add("SCI. ADV. MATER.");
        set.add("ALTERN. THER. HEALTH MED.");
        set.add("CMES-COMP. MODEL. ENG. SCI.");
        set.add("EXP. THER. MED.");
        set.add("FRONT. ENERGY RES.");
        set.add("MATH. BIOSCI. ENG.");
        set.add("TROP. J. PHARM. RES.");
        return set;
    }

    public static Set<String> getEarly2023() {
        Set<String> set = new HashSet<>();
        set.add("JOURNAL OF ENVIRONMENTAL AND PUBLIC HEALTH");
        set.add("9424753");
        set.add("J CLIN LAB ANAL");
        set.add("WORLD JOURNAL OF CLINICAL CASES");
        set.add("101682385");
        set.add("101591609");
        set.add("JOURNAL OF CLINICAL LABORATORY ANALYSIS");
        set.add("TEXTILE RESEARCH JOURNAL");
        set.add("BUILDINGS BASEL");
        set.add("INT J CONTROL AUTOM");
        set.add("MACHINES");
        set.add("J BIOMATER TISSUE ENG");
        set.add("101512861");
        set.add("PSYCHIATRIA DANUBINA");
        set.add("ACTA MEDICA MEDITERRANEA");
        set.add("J STRUCT CHEM");
        set.add("ECON RES EKON ISTRAZ");
        set.add("FRONT ENVIRON SCI");
        set.add("FRONT EARTH SCI");
        set.add("FRONTIERS IN EARTH SCIENCE");
        set.add("9878794");
        set.add("MICROPROCESS MICROSYST");
        set.add("INT J CONTROL AUTOM SYST");
        set.add("EKON ISTRAZ");
        set.add("ENERGY REPORTS");
        set.add("INTERNATIONAL JOURNAL OF ELECTROCHEMICAL SCIENCE");
        set.add("FRONT SURG");
        set.add("101493030");
        set.add("PHARMAZIE");
        set.add("MATH PROBL ENG");
        set.add("INT J ELECTROCHEM SCI");
        set.add("WORLD J CLIN CASES");
        set.add("COMPUTATIONAL AND MATHEMATICAL METHODS IN MEDICINE");
        set.add("AMERICAN JOURNAL OF TRANSLATIONAL RESEARCH");
        set.add("101735833");
        set.add("JOURNAL OF INDUSTRIAL AND MANAGEMENT OPTIMIZATION");
        set.add("INORGANIC AND NANO METAL CHEMISTRY");
        set.add("AEROSPACE BASEL");
        set.add("INTERNATIONAL JOURNAL OF CONTROL AUTOMATION AND SYSTEMS");
        set.add("MOB INF SYST");
        set.add("BUILDINGS");
        set.add("COMPUT MATH METHOD M");
        set.add("101721978");
        set.add("FOOD SCIENCE AND TECHNOLOGY");
        set.add("FOOD SCI TECHNOL");
        set.add("FRONT EARTH SC SWITZ");
        set.add("MATHEMATICAL PROBLEMS IN ENGINEERING");
        set.add("J BIOMATER TISS ENG");
        set.add("JOURNAL OF BIOMATERIALS AND TISSUE ENGINEERING");
        set.add("TEXT RES J");
        set.add("MICROPROCESS MICROSY");
        set.add("ECONOMIC RESEARCH EKONOMSKA ISTRAZIVANJA");
        set.add("PSYCHIATR DANUB");
        set.add("FRONT ENV SCI SWITZ");
        set.add("J STRUCT CHEM+");
        set.add("MOBILE INFORMATION SYSTEMS");
        set.add("ENERGY REP");
        set.add("GEOFLUIDS");
        set.add("FOOD SCI TECH BRAZIL");
        set.add("101516361");
        set.add("AM J TRANSL RES");
        set.add("MICROPROCESSORS AND MICROSYSTEMS");
        set.add("PSYCHIAT DANUB");
        set.add("101748773");
        set.add("INORG NANO MET CHEM");
        set.add("J ENVIRON PUBLIC HEA");
        set.add("INT J ELECTROCHEM SC");
        set.add("101277751");
        set.add("101618806");
        set.add("J IND MANAG OPTIM");
        set.add("8801384");
        set.add("FRONTIERS IN SURGERY");
        set.add("JOURNAL OF STRUCTURAL CHEMISTRY");
        set.add("100954992");
        set.add("101645127");
        set.add("ACTA MEDICA MEDITERR");
        set.add("J ENVIRON PUBLIC HEALTH");
        set.add("AEROSPACE");
        set.add("FRONTIERS IN ENVIRONMENTAL SCIENCE");
        set.add("9800766");
        set.add("COMPUT MATH METHOD MED");
        set.add("101638676");
        set.add("INORGANIC AND NANO-METAL CHEMISTRY");
        set.add("ECONOMIC RESEARCH-EKONOMSKA ISTRAZIVANJA");
        set.add("0040-5175");
        set.add("1468-8115");
        set.add("1547-5816");
        set.add("1024-123X");
        set.add("1748-670X");
        set.add("2352-4847");
        set.add("2470-1556");
        set.add("0022-4766");
        set.add("1452-3981");
        set.add("0141-9331");
        set.add("1598-6446");
        set.add("1574-017X");
        set.add("1331-677X");
        set.add("0101-2061");
        set.add("1687-9805");
        set.add("0031-7144");
        set.add("0353-5053");
        set.add("0393-6384");
        set.add("1943-8141");
        set.add("2157-9083");
        set.add("0887-8013");
        set.add("2307-8960");
        set.add("2296-875X");
        set.add("1746-7748");
        set.add("1468-8123");
        set.add("2296-6463");
        set.add("1553-166X");
        set.add("1563-5147");
        set.add("2226-4310");
        set.add("2075-5309");
        set.add("1748-6718");
        set.add("2075-1702");
        set.add("2470-1564");
        set.add("1573-8779");
        set.add("2296-665X");
        set.add("1872-9436");
        set.add("2005-4092");
        set.add("1875-905X");
        set.add("1848-9664");
        set.add("1678-457X");
        set.add("1687-9813");
        set.add("1849-0867");
        set.add("2283-9720");
        set.add("2157-9091");
        set.add("1098-2825");
        set.add("FRONT EARTH SC-SWITZ");
        set.add("AEROSPACE-BASEL");
        set.add("BUILDINGS-BASEL");
        set.add("INORG NANO-MET CHEM");
        set.add("FRONT ENV SCI-SWITZ");
        set.add("ECON RES-EKON ISTRAZ");
        set.add("FOOD SCI TECH-BRAZIL");
        set.add("TEXT. RES. J.");
        set.add("FRONT. EARTH SCI.");
        set.add("J. IND. MANAG. OPTIM.");
        set.add("MATH. PROBL. ENG.");
        set.add("COMPUT. MATH. METHOD MED.");
        set.add("ENERGY REP.");
        set.add("INORG. NANO-MET. CHEM.");
        set.add("J. STRUCT. CHEM.");
        set.add("INT. J. ELECTROCHEM. SCI.");
        set.add("FRONT. ENVIRON. SCI.");
        set.add("MICROPROCESS. MICROSYST.");
        set.add("INT. J. CONTROL AUTOM. SYST.");
        set.add("MOB. INF. SYST.");
        set.add("EKON. ISTRAZ.");
        set.add("FOOD SCI. TECHNOL.");
        set.add("J. ENVIRON. PUBLIC HEALTH");
        set.add("PSYCHIATR. DANUB.");
        set.add("ACTA MEDICA MEDITERR.");
        set.add("AM. J. TRANSL. RES.");
        set.add("J. BIOMATER. TISSUE ENG.");
        set.add("J. CLIN. LAB. ANAL.");
        set.add("WORLD J. CLIN. CASES");
        set.add("FRONT. SURG.");
        return set;
    }

    public static Set<String> getEarly2021() {
        Set<String> set = new HashSet<>();
        set.add("AIMS MATHEMATICS");
        set.add("FRONT CELL DEV BIOL");
        set.add("101629355");
        set.add("SCI PROGRAMMING NETH");
        set.add("MOL THER NUCL ACIDS");
        set.add("VACCINES");
        set.add("101658402");
        set.add("MICROPROCESS MICROSYST");
        set.add("AGING US");
        set.add("PHARMAZIE");
        set.add("JOURNAL OF CELLULAR AND MOLECULAR MEDICINE");
        set.add("101778895");
        set.add("WORLD J CLIN CASES");
        set.add("J MATH");
        set.add("9891351");
        set.add("ANNALS OF PALLIATIVE MEDICINE");
        set.add("9888589");
        set.add("101586313");
        set.add("101676334");
        set.add("INT J GEN MED");
        set.add("SHOCK AND VIBRATION");
        set.add("AIMS MATH");
        set.add("SHOCK VIB");
        set.add("0346411");
        set.add("101475259");
        set.add("MITOCHONDRIAL DNA PART B RESOURCES");
        set.add("FRONTIERS IN CELL AND DEVELOPMENTAL BIOLOGY");
        set.add("101535920");
        set.add("JOURNAL OF MATHEMATICS");
        set.add("ONCOTARGETS THER");
        set.add("EXPERIMENTAL AND MOLECULAR PATHOLOGY");
        set.add("COMPLEXITY");
        set.add("FRONT MED LAUSANNE");
        set.add("BIOFACTORS");
        set.add("J PERS MED");
        set.add("SCI PROGRAM");
        set.add("9609063");
        set.add("HEALTHCARE");
        set.add("JOURNAL OF INTERNATIONAL MEDICAL RESEARCH");
        set.add("FRONTIERS IN MEDICINE");
        set.add("MOL MED REP");
        set.add("ANN PALLIAT MED");
        set.add("101602269");
        set.add("101585484");
        set.add("101515487");
        set.add("HEALTHCARE BASEL");
        set.add("FRONT MOL BIOSCI");
        set.add("CANCER CELL INT");
        set.add("J MATH UK");
        set.add("CANCER MANAG RES");
        set.add("JOURNAL OF CANCER");
        set.add("WORLD JOURNAL OF CLINICAL CASES");
        set.add("ANN PALLLIAT MED");
        set.add("101512700");
        set.add("MITOCHONDRIAL DNA PART B RESOUR");
        set.add("JOURNAL OF PERSONALIZED MEDICINE");
        set.add("101514322");
        set.add("ADV CIV ENG");
        set.add("INTERNATIONAL JOURNAL OF GENERAL MEDICINE");
        set.add("101691304");
        set.add("BIOLOGY BASEL");
        set.add("9878794");
        set.add("101630250");
        set.add("101508617");
        set.add("CANCER CELL INTERNATIONAL");
        set.add("ONCOTARGETS AND THERAPY");
        set.add("101139795");
        set.add("VACCINES BASEL");
        set.add("0370711");
        set.add("101581621");
        set.add("FRONT MED");
        set.add("EXP MOL PATHOL");
        set.add("DIAGNOSTICS");
        set.add("MOLECULAR THERAPY NUCLEIC ACIDS");
        set.add("CANCER MANAGEMENT AND RESEARCH");
        set.add("101083777");
        set.add("MICROPROCESS MICROSY");
        set.add("MED SCI MONITOR");
        set.add("SCIENTIFIC PROGRAMMING");
        set.add("101648047");
        set.add("MICROPROCESSORS AND MICROSYSTEMS");
        set.add("DISCRETE DYN NAT SOC");
        set.add("MEDICAL SCIENCE MONITOR");
        set.add("ADVANCES IN CIVIL ENGINEERING");
        set.add("8807441");
        set.add("101653173");
        set.add("MOLECULAR MEDICINE REPORTS");
        set.add("101580444");
        set.add("101618806");
        set.add("MITOCHONDRIAL DNA B");
        set.add("FRONTIERS IN MOLECULAR BIOSCIENCES");
        set.add("101666525");
        set.add("BIOMEDICINES");
        set.add("101587988");
        set.add("DISCRETE DYNAMICS IN NATURE AND SOCIETY");
        set.add("9800766");
        set.add("LIFE BASEL");
        set.add("J INT MED RES");
        set.add("J CANCER");
        set.add("J CELL MOL MED");
        set.add("MITOCHONDRIAL DNA PART B-RESOURCES");
        set.add("LIFE-BASEL");
        set.add("BIOLOGY-BASEL");
        set.add("MOLECULAR THERAPY-NUCLEIC ACIDS");
        set.add("AGING-US");
        set.add("1076-2787");
        set.add("1070-9622");
        set.add("1687-8086");
        set.add("0141-9331");
        set.add("1058-9244");
        set.add("0951-6433");
        set.add("2296-634X");
        set.add("1026-0226");
        set.add("2314-4629");
        set.add("0031-7144");
        set.add("2162-2531");
        set.add("0014-4800");
        set.add("1582-1838");
        set.add("1791-2997");
        set.add("0300-0605");
        set.add("1837-9664");
        set.add("1945-4589");
        set.add("1178-6930");
        set.add("1179-1322");
        set.add("2307-8960");
        set.add("2224-5820");
        set.add("1099-0526");
        set.add("1875-9203");
        set.add("1687-8094");
        set.add("2227-9059");
        set.add("1872-9436");
        set.add("1875-919X");
        set.add("1872-8081");
        set.add("2380-2359");
        set.add("2296-889X");
        set.add("2075-1729");
        set.add("2079-7737");
        set.add("1607-887X");
        set.add("2473-6988");
        set.add("2314-4785");
        set.add("1096-0945");
        set.add("1582-4934");
        set.add("1791-3004");
        set.add("1473-2300");
        set.add("1643-3750");
        set.add("1475-2867");
        set.add("2224-5839");
        set.add("1178-7074");
        set.add("2296-858X");
        set.add("2075-4426");
        set.add("2227-9032");
        set.add("2075-4418");
        set.add("2076-393X");
        set.add("SCI PROGRAMMING-NETH");
        set.add("J MATH-UK");
        set.add("MOL THER-NUCL ACIDS");
        set.add("FRONT MED-LAUSANNE");
        set.add("HEALTHCARE-BASEL");
        set.add("VACCINES-BASEL");
        set.add("SHOCK VIB.");
        set.add("ADV. CIV. ENG.");
        set.add("MICROPROCESS. MICROSYST.");
        set.add("SCI. PROGRAM.");
        set.add("MITOCHONDRIAL DNA PART B-RESOUR.");
        set.add("FRONT. MOL. BIOSCI.");
        set.add("FRONT. CELL. DEV. BIOL.");
        set.add("DISCRETE DYN. NAT. SOC.");
        set.add("AIMS MATH.");
        set.add("J. MATH.");
        set.add("MOL. THER.-NUCL. ACIDS");
        set.add("EXP. MOL. PATHOL.");
        set.add("J. CELL. MOL. MED.");
        set.add("MOL. MED. REP.");
        set.add("J. INT. MED. RES.");
        set.add("J. CANCER");
        set.add("MED. SCI. MONITOR");
        set.add("ONCOTARGETS THER.");
        set.add("CANCER MANAG. RES.");
        set.add("CANCER CELL INT.");
        set.add("WORLD J. CLIN. CASES");
        set.add("ANN. PALLLIAT. MED.");
        set.add("INT. J. GEN. MED.");
        set.add("FRONT. MED.");
        set.add("J. PERS. MED.");
        return set;
    }

    public static Set<String> getEarly2020() {
        Set<String> set = new HashSet<>();
        set.add("101531947");
        set.add("9717360");
        set.add("EXP THER MED");
        set.add("INT J ENERG RES");
        set.add("ONCOL RES");
        set.add("J CLIN MED");
        set.add("BIOSCI REP");
        set.add("101776982");
        set.add("ARTIF CELL NANOMED B");
        set.add("INTERNATIONAL JOURNAL OF ENVIRONMENTAL RESEARCH AND PUBLIC HEALTH");
        set.add("101599172");
        set.add("CATALYSTS");
        set.add("WATER");
        set.add("0375521");
        set.add("101580302");
        set.add("101555929");
        set.add("ATMOSPHERE");
        set.add("101670234");
        set.add("AMERICAN JOURNAL OF TRANSLATIONAL RESEARCH");
        set.add("101088195");
        set.add("101695672");
        set.add("8205768");
        set.add("ONCOLOGY REPORTS");
        set.add("MATHEMATICAL PROBLEMS IN ENGINEERING");
        set.add("J CELL BIOCHEM");
        set.add("0346411");
        set.add("J BIOMATER TISS ENG");
        set.add("101475259");
        set.add("ONCOLOGY RESEARCH");
        set.add("9422756");
        set.add("8102797");
        set.add("ZEITSCHRIFT FUR KRISTALLOGRAPHIE NEW CRYSTAL STRUCTURES");
        set.add("MINERALS BASEL");
        set.add("INTERNATIONAL JOURNAL OF CLINICAL AND EXPERIMENTAL PATHOLOGY");
        set.add("BRAZILIAN JOURNAL OF MEDICAL AND BIOLOGICAL RESEARCH");
        set.add("COATINGS");
        set.add("BIOFACTORS");
        set.add("101204366");
        set.add("JOURNAL OF INTERNATIONAL MEDICAL RESEARCH");
        set.add("MOL MED REP");
        set.add("101546871");
        set.add("Z KRIST NEW CRYST ST");
        set.add("INTERNATIONAL JOURNAL OF ENERGY RESEARCH");
        set.add("101606588");
        set.add("ACTA MEDICA MEDITERR");
        set.add("INT J ENV RES PUB HE");
        set.add("EUR REV MED PHARMACOL SCI");
        set.add("INT J CLIN EXP PATHO");
        set.add("APPL SCI BASEL");
        set.add("ADV DIFFER EQU NY");
        set.add("CANCER BIOMARK");
        set.add("101664675");
        set.add("101600173");
        set.add("101739125");
        set.add("101514322");
        set.add("ADV CIV ENG");
        set.add("ACTA MEDICA MEDITERRANEA");
        set.add("ELECTRONICS");
        set.add("ADVANCES IN DIFFERENCE EQUATIONS");
        set.add("AMERICAN JOURNAL OF CANCER RESEARCH");
        set.add("POLYMERS BASEL");
        set.add("BRAZ J MED BIOL RES");
        set.add("SUSTAINABILITY BASEL");
        set.add("ONCOTARGETS AND THERAPY");
        set.add("101167924");
        set.add("EXPERIMENTAL AND THERAPEUTIC MEDICINE");
        set.add("101493030");
        set.add("J NANOSCI NANOTECHNO");
        set.add("INT J ELECTROCHEM SCI");
        set.add("101671521");
        set.add("BIOMED RES INT");
        set.add("101594777");
        set.add("MATERIALS");
        set.add("101721978");
        set.add("JOURNAL OF NANOSCIENCE AND NANOTECHNOLOGY");
        set.add("JOURNAL OF CELLULAR BIOCHEMISTRY");
        set.add("JOURNAL OF BIOMATERIALS AND TISSUE ENGINEERING");
        set.add("MATHEMATICS BASEL");
        set.add("8213295");
        set.add("MEDICINE");
        set.add("101471010");
        set.add("AM J TRANSL RES");
        set.add("BIOSCIENCE REPORTS");
        set.add("MEDICAL SCIENCE MONITOR");
        set.add("ADVANCES IN CIVIL ENGINEERING");
        set.add("101653204");
        set.add("CELLS");
        set.add("101579976");
        set.add("JOURNAL OF CELLULAR PHYSIOLOGY");
        set.add("101531236");
        set.add("101697598");
        set.add("J INT MED RES");
        set.add("101748883");
        set.add("WATER SUI");
        set.add("INT J CLIN EXP PATHOL");
        set.add("LIFE SCIENCES");
        set.add("BIOMED RESEARCH INTERNATIONAL");
        set.add("EUROPEAN REVIEW FOR MEDICAL AND PHARMACOLOGICAL SCIENCES");
        set.add("101591609");
        set.add("SENSORS");
        set.add("PLANTS BASEL");
        set.add("ELECTRONICS SWITZ");
        set.add("METALS");
        set.add("ADV DIFFER EQU");
        set.add("101596181");
        set.add("LIFE SCI");
        set.add("ENERGIES");
        set.add("ATMOSPHERE BASEL");
        set.add("APPLIED SCIENCES BASEL");
        set.add("BOUNDARY VALUE PROBLEMS");
        set.add("DESALIN WATER TREAT");
        set.add("ARTIFICIAL CELLS NANOMEDICINE AND BIOTECHNOLOGY");
        set.add("INTERNATIONAL JOURNAL OF ELECTROCHEMICAL SCIENCE");
        set.add("AGING US");
        set.add("J CELL PHYSIOL");
        set.add("101480565");
        set.add("Z KRIST NEW CRYST STRUCT");
        set.add("9888589");
        set.add("DESALINATION AND WATER TREATMENT");
        set.add("8911335");
        set.add("BOUND VALUE PROBL");
        set.add("101630280");
        set.add("JOURNAL OF INEQUALITIES AND APPLICATIONS");
        set.add("ONCOL LETT");
        set.add("METALS BASEL");
        set.add("101238455");
        set.add("ONCOTARGETS THER");
        set.add("EXPERIMENTAL AND MOLECULAR PATHOLOGY");
        set.add("COMPLEXITY");
        set.add("ONCOLOGY LETTERS");
        set.add("9609063");
        set.add("0050222");
        set.add("EUR REV MED PHARMACO");
        set.add("101613952");
        set.add("INT J ELECTROCHEM SC");
        set.add("INTERNATIONAL JOURNAL OF MOLECULAR MEDICINE");
        set.add("AM J CANCER RES");
        set.add("INT J ENVIRON RES PUBLIC HEALTH");
        set.add("INT J MOL MED");
        set.add("POLYMERS");
        set.add("100964009");
        set.add("ARTIF CELL NANOMED BIOTECHNOL");
        set.add("101633495");
        set.add("101256509");
        set.add("101682385");
        set.add("101666579");
        set.add("101562250");
        set.add("BIOMED PHARMACOTHER");
        set.add("MOLECULES");
        set.add("J BIOMATER TISSUE ENG");
        set.add("INT J ENERGY RES");
        set.add("101545357");
        set.add("INTERNATIONAL JOURNAL OF CLINICAL AND EXPERIMENTAL MEDICINE");
        set.add("9208097");
        set.add("INT J CLIN EXP MED");
        set.add("101508617");
        set.add("MINERALS");
        set.add("SUSTAINABILITY");
        set.add("MATH PROBL ENG");
        set.add("2985248R");
        set.add("101639462");
        set.add("0370711");
        set.add("NATURAL PRODUCT RESEARCH");
        set.add("EXP MOL PATHOL");
        set.add("9810955");
        set.add("101766564");
        set.add("8112917");
        set.add("J NANOSCI NANOTECHNOL");
        set.add("MATHEMATICS");
        set.add("MEDICINE (BALTIMORE)");
        set.add("INTERNATIONAL JOURNAL OF IMMUNOPATHOLOGY AND PHARMACOLOGY");
        set.add("MED SCI MONITOR");
        set.add("JOURNAL OF CLINICAL MEDICINE");
        set.add("BRAZILIAN J MED BIOL RES");
        set.add("8807441");
        set.add("MOLECULAR MEDICINE REPORTS");
        set.add("CANCER BIOMARKERS");
        set.add("J INEQUAL APPL");
        set.add("NAT PROD RES");
        set.add("SYMMETRY BASEL");
        set.add("PROCESSES");
        set.add("INT J IMMUNOPATH PH");
        set.add("INT J IMMUNOPATHOL PHARMACOL");
        set.add("SENSORS BASEL");
        set.add("IEEE ACCESS");
        set.add("AGRONOMY BASEL");
        set.add("BIOMEDICINE & PHARMACOTHERAPY");
        set.add("ONCOL REP");
        set.add("101549944");
        set.add("101769288");
        set.add("BIOSCIENCE REP");
        set.add("CELLS BASEL");
        set.add("APPLIED SCIENCES-BASEL");
        set.add("ZEITSCHRIFT FUR KRISTALLOGRAPHIE-NEW CRYSTAL STRUCTURES");
        set.add("AGRONOMY-BASEL");
        set.add("PLANTS-BASEL");
        set.add("AGING-US");
        set.add("SYMMETRY-BASEL");
        set.add("1533-4880");
        set.add("2169-1401");
        set.add("1687-8086");
        set.add("0363-907X");
        set.add("1024-123X");
        set.add("1076-2787");
        set.add("1944-3994");
        set.add("1452-3981");
        set.add("1478-6419");
        set.add("1433-7266");
        set.add("2169-3536");
        set.add("0730-2312");
        set.add("0021-9541");
        set.add("0144-8463");
        set.add("2314-6133");
        set.add("0951-6433");
        set.add("1687-2770");
        set.add("1687-1847");
        set.add("1029-242X");
        set.add("1128-3602");
        set.add("1936-2625");
        set.add("0025-7974");
        set.add("1940-5901");
        set.add("0753-3322");
        set.add("0014-4800");
        set.add("0100-879X");
        set.add("1574-0153");
        set.add("0394-6320");
        set.add("0965-0407");
        set.add("2156-6976");
        set.add("1792-1074");
        set.add("1792-0981");
        set.add("1178-6930");
        set.add("1021-335X");
        set.add("1791-2997");
        set.add("1107-3756");
        set.add("0300-0605");
        set.add("1943-8141");
        set.add("2157-9083");
        set.add("1945-4589");
        set.add("0024-3205");
        set.add("0393-6384");
        set.add("2075-4701");
        set.add("2079-6412");
        set.add("1996-1944");
        set.add("1533-4899");
        set.add("2075-163X");
        set.add("2073-4433");
        set.add("2169-141X");
        set.add("1687-8094");
        set.add("1099-114X");
        set.add("1563-5147");
        set.add("1424-8220");
        set.add("1996-1073");
        set.add("2076-3417");
        set.add("2073-4360");
        set.add("2079-9292");
        set.add("2227-9717");
        set.add("1099-0526");
        set.add("1944-3986");
        set.add("2073-4344");
        set.add("1420-3049");
        set.add("1478-6427");
        set.add("2197-4578");
        set.add("2071-1050");
        set.add("2073-4441");
        set.add("2073-4395");
        set.add("1097-4644");
        set.add("1097-4652");
        set.add("1573-4935");
        set.add("1872-8081");
        set.add("2223-7747");
        set.add("2073-4409");
        set.add("2227-7390");
        set.add("1536-5964");
        set.add("1950-6007");
        set.add("1096-0945");
        set.add("1414-431X");
        set.add("1875-8592");
        set.add("2058-7384");
        set.add("1555-3906");
        set.add("1643-3750");
        set.add("1792-1082");
        set.add("1792-1015");
        set.add("1791-2431");
        set.add("1791-3004");
        set.add("1791-244X");
        set.add("1473-2300");
        set.add("2157-9091");
        set.add("1879-0631");
        set.add("2077-0383");
        set.add("1660-4601");
        set.add("2283-9720");
        set.add("2073-8994");
        set.add("METALS-BASEL");
        set.add("MINERALS-BASEL");
        set.add("ATMOSPHERE-BASEL");
        set.add("SENSORS-BASEL");
        set.add("APPL SCI-BASEL");
        set.add("POLYMERS-BASEL");
        set.add("ELECTRONICS-SWITZ");
        set.add("Z KRIST-NEW CRYST ST");
        set.add("SUSTAINABILITY-BASEL");
        set.add("WATER-SUI");
        set.add("CELLS-BASEL");
        set.add("ADV DIFFER EQU-NY");
        set.add("MATHEMATICS-BASEL");
        set.add("J. NANOSCI. NANOTECHNOL.");
        set.add("ARTIF. CELL. NANOMED. BIOTECHNOL.");
        set.add("ADV. CIV. ENG.");
        set.add("INT. J. ENERGY RES.");
        set.add("MATH. PROBL. ENG.");
        set.add("APPL. SCI.-BASEL");
        set.add("DESALIN. WATER TREAT.");
        set.add("INT. J. ELECTROCHEM. SCI.");
        set.add("NAT. PROD. RES.");
        set.add("Z. KRIST.-NEW CRYST. STRUCT.");
        set.add("J. CELL. BIOCHEM.");
        set.add("J. CELL. PHYSIOL.");
        set.add("BIOSCI. REP.");
        set.add("BIOMED RES. INT.");
        set.add("BOUND. VALUE PROBL.");
        set.add("ADV. DIFFER. EQU.");
        set.add("J. INEQUAL. APPL.");
        set.add("EUR. REV. MED. PHARMACOL. SCI.");
        set.add("INT. J. CLIN. EXP. PATHOL.");
        set.add("INT. J. CLIN. EXP. MED.");
        set.add("BIOMED. PHARMACOTHER.");
        set.add("EXP. MOL. PATHOL.");
        set.add("BRAZILIAN J. MED. BIOL. RES.");
        set.add("CANCER BIOMARK.");
        set.add("INT. J. IMMUNOPATHOL. PHARMACOL.");
        set.add("ONCOL. RES.");
        set.add("AM. J. CANCER RES.");
        set.add("MED. SCI. MONITOR");
        set.add("ONCOL. LETT.");
        set.add("EXP. THER. MED.");
        set.add("ONCOTARGETS THER.");
        set.add("ONCOL. REP.");
        set.add("MOL. MED. REP.");
        set.add("INT. J. MOL. MED.");
        set.add("J. INT. MED. RES.");
        set.add("AM. J. TRANSL. RES.");
        set.add("J. BIOMATER. TISSUE ENG.");
        set.add("LIFE SCI.");
        set.add("J. CLIN. MED.");
        set.add("INT. J. ENVIRON. RES. PUBLIC HEALTH");
        set.add("ACTA MEDICA MEDITERR.");
        return set;
    }

    public static List<En.C1Auth> upDataC1Au(String c1Au, Map<String, List<String>> bestAdMap) {
        if (null != c1Au && c1Au.contains("sort")) {
            List<En.C1Auth> c1Auths = JSONObject.parseArray(c1Au, En.C1Auth.class);
            for (En.C1Auth c1Auth : c1Auths) {
                String full = c1Auth.getFull();
                if (bestAdMap.containsKey(full)) {
                    List<String> list = bestAdMap.get(full);
                    if (list.size() == 1) {
                        c1Auth.setFull(full + "ĺļ" + list.get(0));
                        bestAdMap.remove(full);
                    } else if (list.size() > 1) {
                        c1Auth.setFull(full + "ĺļ" + list.get(0));
                        list.remove(0);
                        bestAdMap.put(full, list);
                    }
                }
            }
            if (!bestAdMap.isEmpty()) {
                List<String> auList = new ArrayList<>();
                Set<String> infoSet = new HashSet<>();
                for (Map.Entry<String, List<String>> entry : bestAdMap.entrySet()) {
                    auList.add(entry.getKey());
                    infoSet.addAll(entry.getValue());
                }
                En.C1Auth c1Auth = new En.C1Auth();
                c1Auth.setAuthList(auList);
                c1Auth.setFull(CollUtil.join(infoSet, "ĺļ"));
                c1Auths.add(c1Auth);
            }
            return c1Auths;
        }
        return new ArrayList<>();
    }

    public static Map<String, List<String>> bestAd(List<String> aList, List<String> bList) {
        // 把pm的ad信息，添加到ut的c1中去。第一步，两两无差别全组合，其中，a为来自pm的，b为来自ut的，将结果全部存入abAndScoreMap中。
        Map<List<String>, Double> abAndScoreMap = new HashMap<>();
        for (String a : aList) {
            if (a.length() > 8) {
                for (String b : bList) {
                    if (b.length() > 8) {
                        // 计算得分时，因pm中的中国是China，而ut中的中国是Peoples R China，需要给予转换，但存入结果时，仍使用原始的，这样可以确保后续仍能找到。
                        String f = b.replaceAll("(?i)Peoples R China", "China").replaceAll("(?i)People's Republic of China", "China")
                                .replaceAll("(?i)P\\.R\\. China", "China").replaceAll("(?i)PR China", "China").replaceAll("(?i)United States of America", "USA");
                        // pm的数据中往往存在电子邮箱和"Electronic address:"的字符串，比对前给予删除。
                        double score = calculateCosineSimilarity(getMap(a.replaceAll("([a-zA-Z-_0-9]+@[A-Za-z0-9-]+\\.[A-Za-z0-9.]{2,})", " ").replaceAll("(?i)Electronic address:", "")), getMap(f));
                        List<String> list = new ArrayList<>();
                        list.add(a);
                        list.add(b);
                        abAndScoreMap.put(list, score);
                    }
                }
            }
        }

        // 第二步，循环aList（为什么循环aList而不是bList？因为理论上我们需要每一个a均有一个对应的b，即a往b里面映射，形成baMap，最多aList.size()组，所以可以循环aList），每次循环均求两两组合中得分最高的，而且一旦某组被保存进了结果baMap，下次的一组即不可再保存了（使用set来实现）。
        Map<String, List<String>> baMap = new HashMap<>();
        Set<String> set = new HashSet<>();
        double allMax = pubBaMap(baMap, aList, abAndScoreMap, set, 0.35);
        if (aList.size() > baMap.size() && allMax > 0.6) {
            pubBaMap(baMap, aList, abAndScoreMap, set, 0.25);
        }
        // 第三步，如果aList.size() > baMap.size()，说明有少的，可能是得分不足0.5，或者是来自于ut的b少于来自于pm的a，此时对于少的，按未知[Anonymous]并加上序号给put到结果baMap中，后续使用时，单独给添加进去。但为什么要!baMap.isEmpty()？原因是如果没有一个得分大于0.5，说明有可能不是同一篇论文，所以不能硬添加进去。
        if (aList.size() > baMap.size() && !baMap.isEmpty()) {
            for (int i = 0; i < aList.size(); i++) {
                String a = aList.get(i);
                boolean bool = true;
                for (Map.Entry<String, List<String>> entry : baMap.entrySet()) {
                    if (entry.getValue().contains(a)) {
                        bool = false;
                        break;
                    }
                }
                // 如果bool为true，说明a没有匹配到最佳的b，需要进一步筛选。如果筛选出来的b，符合条件(allMax > 0.7 && max > 0.25) || max > 0.5，添加到baMap，如果不符合条件，则b给予未知（"[Anonymous]_" + i）
                if (bool) {
                    double max = 0.0;
                    String best = "";
                    for (String b : bList) {
                        String f = b.replaceAll("(?i)Peoples R China", "China").replaceAll("(?i)People's Republic of China", "China")
                                .replaceAll("(?i)P\\.R\\. China", "China").replaceAll("(?i)PR China", "China").replaceAll("(?i)United States of America", "USA");
                        double score = calculateCosineSimilarity(getMap(a.replaceAll("([a-zA-Z-_0-9]+@[A-Za-z0-9-]+\\.[A-Za-z0-9.]{2,})", " ").replaceAll("(?i)Electronic address:", "")), getMap(f));
                        if (score > max) {
                            max = score;
                            best = b;
                        }
                    }
                    if ((allMax > 0.6 && max > 0.25) || max > 0.35) {
                        if (baMap.size() >= bList.size()) {
                            pubStrStrListMap(baMap, a, ("[Anonymous]_" + i));
                        } else {
                            pubStrStrListMap(baMap, a, best);
                        }
                    } else {
                        pubStrStrListMap(baMap, a, ("[Anonymous]_" + i));
                    }
                }
            }
        }
        return allMax > 0.35 ? baMap : new HashMap<>();
    }

    public static void pubStrStrListMap(Map<String, List<String>> baMap, String a, String b) {
        if (baMap.containsKey(b)) {
            List<String> list = baMap.get(b);
            if (!list.contains(a)) {
                list.add(a);
                baMap.put(b, list);
            }
        } else {
            List<String> list = new ArrayList<>();
            list.add(a);
            baMap.put(b, list);
        }
    }

    public static double pubBaMap(Map<String, List<String>> baMap, List<String> aList, Map<List<String>, Double> abAndScoreMap, Set<String> set, double threshold) {
        double allMax = 0.0;
        int number = aList.size() - baMap.size();
        for (int i = 0; i < number; i++) {
            String a = "";
            String b = "";
            double max = 0.0;
            for (Map.Entry<List<String>, Double> entry : abAndScoreMap.entrySet()) {
                if (entry.getValue() > max && entry.getValue() > threshold) {
                    if (!set.contains(entry.getKey().get(0)) && !set.contains(entry.getKey().get(1))) {
                        max = entry.getValue();
                        a = entry.getKey().get(0);
                        b = entry.getKey().get(1);
                    }
                }
            }
            if (max > threshold && !b.isEmpty()) {
                // 注意，这里put进去的是b相对应的a，用于后续b能get出来a。
                pubStrStrListMap(baMap, a, b);
                set.add(a);
                set.add(b);
            }
            allMax = Math.max(max, allMax);
        }
        return allMax;
    }

    public static List<String> getC1Ad(String c1Au) {
        List<String> adList = new ArrayList<>();
        if (null != c1Au && c1Au.contains("sort")) {
            List<En.C1Auth> c1Auths = JSONObject.parseArray(c1Au, En.C1Auth.class);
            for (En.C1Auth c1Auth : c1Auths) {
                if (!adList.contains(c1Auth.getFull()) && c1Auth.getFull().length() > 8) {
                    adList.add(c1Auth.getFull());
                }
            }
        }
        return adList;
    }

    public static List<String> getPmAd(String authorExtend) {
        List<String> adList = new ArrayList<>();
        if (null != authorExtend && authorExtend.contains("Affiliation")) {
            JSONArray objects = JSONUtil.parseArray(authorExtend);
            for (Object object : objects) {
                JSON parse = JSONUtil.parse(object);
                String affiliation = parse.getByPath("Affiliation").toString();
                List<String> infoList = com.alibaba.fastjson.JSON.parseArray(affiliation, String.class);
                for (String s : infoList) {
                    if (!adList.contains(s) && s.length() > 8) {
                        adList.add(s);
                    }
                }
            }
        }
        return adList;
    }

    public static Map<String, Integer> getMap(String strings) {
        String[] split = strings.split(" ");
        Map<String, Integer> map = new HashMap<>();
        for (String s : split) {
            map.put(s, 1);
        }
        return map;
    }

    public static double calculateCosineSimilarity(Map<String, Integer> vectorA, Map<String, Integer> vectorB) {
        // 计算向量的内积
        double dotProduct = 0.0;
        for (Map.Entry<String, Integer> entry : vectorA.entrySet()) {
            String term = entry.getKey();
            int frequencyA = entry.getValue();
            if (vectorB.containsKey(term)) {
                int frequencyB = vectorB.get(term);
                dotProduct += frequencyA * frequencyB;
            }
        }

        // 计算向量的模长
        double normA = calculateVectorNorm(vectorA);
        double normB = calculateVectorNorm(vectorB);

        // 计算余弦相似度
        double cosineSimilarity;
        if (normA != 0 && normB != 0) {
            cosineSimilarity = dotProduct / (normA * normB);
        } else {
            cosineSimilarity = 0.0;
        }

        return cosineSimilarity;
    }

    public static double calculateVectorNorm(Map<String, Integer> vector) {
        double norm = 0.0;
        for (Map.Entry<String, Integer> entry : vector.entrySet()) {
            int frequency = entry.getValue();
            norm += Math.pow(frequency, 2);
        }
        norm = Math.sqrt(norm);
        return norm;
    }


}
